# -*- coding: utf-8 -*-
import json
from time import sleep

import requests,re
import scrapy

from doubanscrapy.items import DoubanscrapyItem
from doubanscrapy.middlewares import IPProxyDownloadMiddleware


class DoubanDemoSpider(scrapy.Spider):
    name = 'douban_demo'
    allowed_domains = ['movie.douban.com']
    # start_urls = ['https://movie.douban.com/']

    def start_requests(self):
        for year in [2021,2020,2019,2018]:
            for i in range(0, 10000, 20):
                url = "https://movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=%E7%94%B5%E5%BD%B1&start=" + str(
                    i) + "&year_range=" + str(year) + "," + str(year)
                print(url)
                yield scrapy.Request(url=url,meta={'url':url},callback=self.parse)
                sleep(2)


    def parse(self, response):
        print(response.text)
        j=[]
        if "检测到有异常请求从您的IP发出" in response.text:
            sleep(10)
            yield scrapy.Request(url=response.meta['url'],meta={'url':response.meta['url']},callback=self.parse)
            print("ok")
        else:
            j = json.loads(response.text)
        print(len(j))
        for data in j['data']:
            meta_section = {'movie_name':data['title'],'director':data['directors'][0],'score':data['rate']}
            yield scrapy.Request(url=data['url'],meta=meta_section,callback=self.parse_item)
            sleep(2)
    def parse_item(self,response):
        html = response.text

        movie_name = response.meta['movie_name']

        director = response.meta['director']

        score = response.meta['score']

        types = "/".join(re.findall(r'<span property="v:genre">([\s\S]*?)</span>', html)) if len(
            re.findall(r'<span property="v:genre">([\s\S]*?)</span>', html)) != 0 else "无"

        area = "/".join(re.findall(r'制片国家/地区:</span> ([\s\S]*?)<br/>', html)).replace(" ","") if len(
            re.findall(r'制片国家/地区:</span> ([\s\S]*?)<br/>', html)) != 0 else "无"

        release = "/".join(re.findall(r'<span property="v:initialReleaseDate" content="[\s\S]*?">([\s\S]*?)\([\s\S]*?\)</span>',
                         html)) if len(re.findall(r'<span property="v:initialReleaseDate" content="[\s\S]*?">([\s\S]*?)\([\s\S]*?\)</span>', html)) != 0 else "无"

        length = re.findall(r'<span property="v:runtime" content="[\s\S]*?">([\s\S]*?)</span>', html)[0] if len(
            re.findall(r'<span property="v:runtime" content="[\s\S]*?">([\s\S]*?)</span>', html)) != 0 else "无"

        comments = re.findall(r'<span property="v:votes">([\s\S]*?)</span>', html)[0] if len(
            re.findall(r'<span property="v:votes">([\s\S]*?)</span>', html)) != 0 else "无"

        item = DoubanscrapyItem(a_movie_name=movie_name,b_director=director,g_score=score,c_types=types,d_area=area,e_release=release,f_length=length,h_comments=comments)
        yield item


